package com.spider.ask120.webmagic.task;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.StringUtils;
import org.jdiy.core.Rs;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.alibaba.fastjson.JSONObject;
import com.spider.ask120.service.ArticleService;
import com.spider.ask120.service.ChannelArticleInfoService;
import com.spider.ask120.service.ChannelService;
import com.spider.ask120.webmagic.Base;
import com.spider.utils.DateUtil;
import com.spider.utils.HttpClientUtil;
import com.spider.utils.PinyinUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.processor.PageProcessor;


/**
 * 描        述：爬数据-健康咨询(频道)
 * 创建时间：2016-8-15
 * @author Jibaole
 */
public class PinDao extends Base implements PageProcessor {
	public static final String URL_INDEX = "http://m.120ask.com/healthart/wx_art_home_all/";
	public static final String URL_LIST = "/?kswys_wx=1&from=atr_home_all&ch_id=[0-9]+";
	public static final String URL_tag_list = "/healthart/wx_lists/\\?tid=[0-9]+";
	public static final String URL_tag_list_cont = "/cron/wxartlist/jump";
	int num=0;
	List<String> change_Ids = new ArrayList<String>();//频道id
	List<String> tag_Ids = new ArrayList<String>();//标签id
	List<String> cont_idS = new ArrayList<String>();//内容id


	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}


	@Override
	public void process(Page page) {
		String pic_dir = fun.getProValue(PINDAO_PIC_FILE_PATH);
		try {
			//列表页
			if (page.getUrl().regex(URL_INDEX+"$").match()) {
				System.err.println("#########################>>>频道页<<<<###################################");
				int m = 1;
				while (true) {
					String url = page.getHtml().xpath("//ul[@class='ChanelList']/a[@href]["+m+"]/@href").get();
					if (null == url) {
						break;
					}
					String name = page.getHtml().xpath("//ul[@class='ChanelList']/a["+m+"]/li/p/b/text()").get();
					String abst = page.getHtml().xpath("//ul[@class='ChanelList']/a["+m+"]/li/p/span/text()").get();
					if ("健康饮食".equals(name) || "民间偏方".equals(name) || "健康经验".equals(name)) {
						m++;
						continue;
					}
					m++;
					//System.out.println(name+"------"+abst);
					String str_id = url.substring(url.indexOf("/wx") + 1, url.indexOf("/?"));
					Rs ch = ChannelService.getChannelByStrIdAndPid(str_id, PINDAO_CHANNEL_PID+"");
					String ch_id = "";
					if (ch.isNull()) {
						Rs channel = new Rs("channel");
						channel.set("name", name);
						channel.set("pinyin", PinyinUtil.converterToFirstSpell(name));
						channel.set("abst", abst);
						channel.set("str_id", str_id);
						channel.set("level", 1);
						channel.set("parent_id", PINDAO_CHANNEL_PID);
						ChannelService.save(channel);
					}else {
						ch_id = ch.get("id");
					}
					//System.out.println(url+"&ch_id="+ch_id);
					page.addTargetRequest(url+"&ch_id="+ch_id);
				}
			} else if(page.getUrl().regex(URL_LIST+"$").match()) {//tag 标签
				System.err.println("#########################>>>标签页<<<<###################################");
				String url = page.getUrl().get();
				String channel_id = url.substring(url.indexOf("ch_id=")+6,url.length());
				//得到所有标签
				int m = 1;
				while(true){
					String name = page.getHtml().xpath("//div[@class='ChanelHomeBox']/ul/li["+m+"]/a/text()").get();
					if (null == name) {
						break;
					}
					String href = page.getHtml().xpath("//div[@class='ChanelHomeBox']/ul/li["+m+"]/a/@href").get();
					m++;
					//System.out.println(name+"-----"+href);
					if (href.indexOf("tid") > 0) {
						String str_id = href.substring(href.indexOf("=")+1, href.indexOf("&"));
						Rs ch = ChannelService.getChannelByStrIdAndPid(str_id, channel_id);
						String ch_id = "";
						if (ch.isNull()) {
							Rs channel = new Rs("channel");
							channel.set("name", name);
							channel.set("pinyin", PinyinUtil.converterToFirstSpell(name));
							channel.set("parent_id", channel_id);
							channel.set("str_id", str_id);
							channel.set("level", 2);
							ChannelService.save(channel);
							ch_id = channel.get("id");
						}else {
							ch_id = ch.get("id");
						}
						//System.out.println(href+"&ch_id="+ch_id);
						page.addTargetRequest(href+"&ch_id="+ch_id);
						//测试 只加载一个标签
//						break;
					}
				}
			}else if(page.getUrl().regex(URL_tag_list).match()){
				System.err.println("#########################>>>详细内容，列表页<<<<###################################");
				String url = page.getUrl().get();
				String channel_id = url.substring(url.indexOf("ch_id=")+6,url.length());
				Rs campRs = ChannelService.getChannelById(channel_id);
				String tid=url.substring(url.indexOf("=")+1, url.indexOf("&"));
				
				Rs ca = ChannelArticleInfoService.getInfoByChId(Integer.parseInt(channel_id));
				Integer record_str_id = ca.getInt(ARTICLE_STR_ID);
				Integer bigger_str_id = record_str_id;
				
				//无下级栏目，开始加载分页数据列表，获取正文URL
				Map<String, String> params = new HashMap<String, String>();
				params.put("tid",tid);
				params.put("tj_type", "5");
				params.put("order", "1");
				Integer count = 1;
				int pageIndex = 1;	
				list:
				while(true){ 
					params.put("page", pageIndex+"");
					//System.out.println("第"+pageIndex+"页");
					String listJson = HttpClientUtil.getInstance().sendHttpPost("http://m.120ask.com/healthart/wx_art_more/", params);
					if(!StringUtils.isEmpty(listJson)){
						try {
							Document doc = Jsoup.parse(listJson);
							//对elements重新排序
							List<Element> eles = sort(doc.getElementsByAttribute("href"));
							//遍历标签
							for (Element link : eles) {
								String linkUrl = link.attr("href");
								String pic_url = link.select("img").attr("src");
								String str_id = linkUrl.substring(linkUrl.indexOf("id=")+3,linkUrl.indexOf("&"));
								
								//如果文章Id比数据库中存储的最新文章Id小 跳出循环
								Integer current_article_str_id = Integer.parseInt(str_id); 
								if (record_str_id >= current_article_str_id) {
									break list;
								}
								
								Rs art = ArticleService.getArticleByStrId(channel_id, 
										link.select("dd").get(0).text(), str_id);
								String a_id = "";
								if (art.isNull()) {
									//下载图片到本地并返回图片地址
									String pic_path = fun.downloadPic(pic_url,pic_dir+"list/"+campRs.getString("pinyin")+"/");
									Rs cont= new Rs("article");
									cont.set("str_id", str_id);
									cont.set("channel_id", channel_id);
									cont.set("title", link.select("dd").get(0).text());
									cont.set("pic", pic_path);
									cont.set("abst", link.select("dd").get(1).text());
									ArticleService.saveArticle(cont);
									a_id = cont.get("id");
								}else {
									a_id = art.get("id");
								}
								
								//System.out.println();
								page.addTargetRequest(linkUrl+"a_id="+a_id);
								
								//记录最新文章str_id
								if (count == 1) {
									bigger_str_id = current_article_str_id;
									count = 2;
								}
							}
						} catch (Exception e) { 
							e.printStackTrace();
						}
					}else{
						break;
					}
					pageIndex ++;
				}
				ca.set(ARTICLE_STR_ID, bigger_str_id);
				ChannelArticleInfoService.save(ca);
			}else if(page.getUrl().toString().indexOf(URL_tag_list_cont) > 0){//详细内容页
				System.err.println("#########################>>>详细内容页<<<<###################################");
				String linkUrl = page.getUrl().toString();
				String tid=linkUrl.substring(linkUrl.indexOf("=")+1, linkUrl.indexOf("&"));
				String a_id = linkUrl.substring(linkUrl.indexOf("a_id=")+5, linkUrl.length());
				Rs cont = ArticleService.getArticleById(a_id);
				cont.set("str_id",tid);
				String time = page.getHtml().xpath("//dd[@class='article_cont-dd2']/span[1]/text()").toString();
				String content = page.getHtml().xpath("//div[@class='article_cont-divp']/html()").toString();
				String pic = page.getHtml().xpath("//div[@class='article_cont-div']/span/img/@data-original").get();
				String pic_path = fun.downloadPic(pic, pic_dir+"article/"+DateUtil.monthDateT(time)+"/");
				JSONObject obj = new JSONObject();
				obj.put("pic_path", pic_path);
				obj.put("cont", content);
				cont.set("content", obj.toJSONString());
				cont.set("create_time", time);
				String  browse_count = page.getHtml().xpath("//dd[@class='article_cont-dd2']/span[2]").toString();
				if(!StringUtils.isEmpty(browse_count)){
					if(browse_count.indexOf("(") >0 && browse_count.indexOf(")") >0){
						cont.set("browse_count", browse_count.substring(browse_count.indexOf("(")+1, browse_count.indexOf(")")));	
					}
				}
				ArticleService.saveArticle(cont);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void runPindao(){
		String url = "http://m.120ask.com/healthart/wx_art_home_all/";
		Spider s = Spider.create(new PinDao()).addUrl(url); 
		s.thread(1).start();
	}
	
	public static void main(String[] args) throws Exception {
//        Function fun = new Function();
//		String spider_scheduler_path = fun.getProValue(PINDAO_SCHEDULER_PATH);
//		FileCacheQueueScheduler scheduler = new FileCacheQueueScheduler(spider_scheduler_path);
//		http://m.120ask.com/healthart/wx_art_home_all/
//		Spider s = Spider.create(new PinDao()).addUrl("http://m.120ask.com/healthart/wx_lists/?tid=704&type=2&ch_id=2688").scheduler(scheduler); 
//		SpiderMonitor.instance().register(s);
//		s.thread(1).start();
		
		
		
		
//		http://m.120ask.com/healthart/wx_lists/?tid=704&type=2&ch_id=2688
		Spider s = Spider.create(new PinDao()).addUrl("http://m.120ask.com/healthart/wx_lists/?tid=704&type=2&ch_id=2688"); 
		SpiderMonitor.instance().register(s);
		s.thread(1).start();
		
		
	}

}
